{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "cac06555-9ce8-4f01-bbef-3f8407f4b54d",
   "metadata": {},
   "source": [
    "# Lab 3: Building custom data connectors for MemGPT\n",
    "This example notebook goes over how to create a connector to load external data sources into MemGPT agents. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "11900885-4d59-4863-8cb3-20ed7f4f570a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pprint import pprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7ccd43f2-164b-4d25-8465-894a3bb54c4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from memgpt import create_client \n",
    "\n",
    "client = create_client() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9a28e38a-7dbe-4530-8260-202322a8458e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[LLMConfig(model='gpt-4', model_endpoint_type='openai', model_endpoint='https://api.openai.com/v1', model_wrapper=None, context_window=8192)]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client.list_models()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "001b35ed-fd81-466c-bc21-84b7f7c71aac",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3.24\n",
      "0.5.0\n"
     ]
    }
   ],
   "source": [
    "import memgpt\n",
    "import chromadb\n",
    "\n",
    "print(memgpt.__version__)\n",
    "print(chromadb.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "94576f44-1156-4c27-ae78-fd2d114b73dd",
   "metadata": {},
   "source": [
    "### Loading external data into archival memory \n",
    "In this section, we'll how you how you can use the `llama-index` library add external data sources as memories into MemGPT. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4a42a2fe-d208-43a3-9210-cbeef0cdb035",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: llama-index in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (0.10.27)\n",
      "Collecting llama-index-readers-web\n",
      "  Downloading llama_index_readers_web-0.2.2-py3-none-any.whl.metadata (1.2 kB)\n",
      "Requirement already satisfied: llama-index-agent-openai<0.3.0,>=0.1.4 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index) (0.2.2)\n",
      "Requirement already satisfied: llama-index-cli<0.2.0,>=0.1.2 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index) (0.1.11)\n",
      "Requirement already satisfied: llama-index-core<0.11.0,>=0.10.27 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index) (0.10.27)\n",
      "Requirement already satisfied: llama-index-embeddings-openai<0.2.0,>=0.1.5 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index) (0.1.7)\n",
      "Requirement already satisfied: llama-index-indices-managed-llama-cloud<0.2.0,>=0.1.2 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index) (0.1.5)\n",
      "Requirement already satisfied: llama-index-legacy<0.10.0,>=0.9.48 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index) (0.9.48)\n",
      "Requirement already satisfied: llama-index-llms-openai<0.2.0,>=0.1.13 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index) (0.1.14)\n",
      "Requirement already satisfied: llama-index-multi-modal-llms-openai<0.2.0,>=0.1.3 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index) (0.1.4)\n",
      "Requirement already satisfied: llama-index-program-openai<0.2.0,>=0.1.3 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index) (0.1.5)\n",
      "Requirement already satisfied: llama-index-question-gen-openai<0.2.0,>=0.1.2 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index) (0.1.3)\n",
      "Requirement already satisfied: llama-index-readers-file<0.2.0,>=0.1.4 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index) (0.1.13)\n",
      "Requirement already satisfied: llama-index-readers-llama-parse<0.2.0,>=0.1.2 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index) (0.1.4)\n",
      "Requirement already satisfied: aiohttp<4.0.0,>=3.9.1 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index-readers-web) (3.9.3)\n",
      "Requirement already satisfied: beautifulsoup4<5.0.0,>=4.12.3 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index-readers-web) (4.12.3)\n",
      "Collecting chromedriver-autoinstaller<0.7.0,>=0.6.3 (from llama-index-readers-web)\n",
      "  Using cached chromedriver_autoinstaller-0.6.4-py3-none-any.whl.metadata (2.1 kB)\n",
      "Collecting html2text<2025.0.0,>=2024.2.26 (from llama-index-readers-web)\n",
      "  Using cached html2text-2024.2.26-py3-none-any.whl\n",
      "INFO: pip is looking at multiple versions of llama-index-readers-web to determine which version is compatible with other requirements. This could take a while.\n",
      "Collecting llama-index-readers-web\n",
      "  Using cached llama_index_readers_web-0.2.1-py3-none-any.whl.metadata (1.2 kB)\n",
      "  Using cached llama_index_readers_web-0.2.0-py3-none-any.whl.metadata (1.2 kB)\n",
      "  Using cached llama_index_readers_web-0.1.23-py3-none-any.whl.metadata (1.2 kB)\n",
      "Collecting newspaper3k<0.3.0,>=0.2.8 (from llama-index-readers-web)\n",
      "  Using cached newspaper3k-0.2.8-py3-none-any.whl.metadata (11 kB)\n",
      "Collecting playwright<2.0,>=1.30 (from llama-index-readers-web)\n",
      "  Using cached playwright-1.46.0-py3-none-macosx_11_0_universal2.whl.metadata (3.5 kB)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.31.0 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index-readers-web) (2.31.0)\n",
      "Collecting selenium<5.0.0,>=4.17.2 (from llama-index-readers-web)\n",
      "  Downloading selenium-4.24.0-py3-none-any.whl.metadata (7.1 kB)\n",
      "Collecting spider-client<0.0.28,>=0.0.27 (from llama-index-readers-web)\n",
      "  Using cached spider_client-0.0.27-py3-none-any.whl\n",
      "Requirement already satisfied: urllib3>=1.1.0 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index-readers-web) (2.2.1)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.9.1->llama-index-readers-web) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.9.1->llama-index-readers-web) (23.2.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.9.1->llama-index-readers-web) (1.4.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.9.1->llama-index-readers-web) (6.0.5)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.9.1->llama-index-readers-web) (1.9.4)\n",
      "Requirement already satisfied: soupsieve>1.2 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from beautifulsoup4<5.0.0,>=4.12.3->llama-index-readers-web) (2.5)\n",
      "Requirement already satisfied: packaging>=23.1 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from chromedriver-autoinstaller<0.7.0,>=0.6.3->llama-index-readers-web) (24.0)\n",
      "Requirement already satisfied: openai>=1.14.0 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index-agent-openai<0.3.0,>=0.1.4->llama-index) (1.16.2)\n",
      "Requirement already satisfied: PyYAML>=6.0.1 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.27->llama-index) (6.0.1)\n",
      "Requirement already satisfied: SQLAlchemy>=1.4.49 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.11.0,>=0.10.27->llama-index) (2.0.29)\n",
      "Requirement already satisfied: dataclasses-json in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.27->llama-index) (0.6.4)\n",
      "Requirement already satisfied: deprecated>=1.2.9.3 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.27->llama-index) (1.2.14)\n",
      "Requirement already satisfied: dirtyjson<2.0.0,>=1.0.8 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.27->llama-index) (1.0.8)\n",
      "Requirement already satisfied: fsspec>=2023.5.0 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.27->llama-index) (2024.2.0)\n",
      "Requirement already satisfied: httpx in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.27->llama-index) (0.25.2)\n",
      "Requirement already satisfied: llamaindex-py-client<0.2.0,>=0.1.16 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.27->llama-index) (0.1.16)\n",
      "Requirement already satisfied: nest-asyncio<2.0.0,>=1.5.8 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.27->llama-index) (1.6.0)\n",
      "Requirement already satisfied: networkx>=3.0 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.27->llama-index) (3.3)\n",
      "Requirement already satisfied: nltk<4.0.0,>=3.8.1 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.27->llama-index) (3.8.1)\n",
      "Requirement already satisfied: numpy in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.27->llama-index) (1.26.4)\n",
      "Requirement already satisfied: pandas in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.27->llama-index) (2.2.1)\n",
      "Requirement already satisfied: pillow>=9.0.0 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.27->llama-index) (10.3.0)\n",
      "Requirement already satisfied: tenacity<9.0.0,>=8.2.0 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.27->llama-index) (8.2.3)\n",
      "Requirement already satisfied: tiktoken>=0.3.3 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.27->llama-index) (0.5.2)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.66.1 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.27->llama-index) (4.66.2)\n",
      "Requirement already satisfied: typing-extensions>=4.5.0 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.27->llama-index) (4.11.0)\n",
      "Requirement already satisfied: typing-inspect>=0.8.0 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.27->llama-index) (0.9.0)\n",
      "Requirement already satisfied: wrapt in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.27->llama-index) (1.16.0)\n",
      "Requirement already satisfied: pymupdf<2.0.0,>=1.23.21 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index-readers-file<0.2.0,>=0.1.4->llama-index) (1.24.1)\n",
      "Requirement already satisfied: pypdf<5.0.0,>=4.0.1 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index-readers-file<0.2.0,>=0.1.4->llama-index) (4.1.0)\n",
      "Requirement already satisfied: striprtf<0.0.27,>=0.0.26 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index-readers-file<0.2.0,>=0.1.4->llama-index) (0.0.26)\n",
      "Requirement already satisfied: llama-parse<0.5.0,>=0.4.0 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llama-index-readers-llama-parse<0.2.0,>=0.1.2->llama-index) (0.4.0)\n",
      "Collecting cssselect>=0.9.2 (from newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web)\n",
      "  Using cached cssselect-1.2.0-py2.py3-none-any.whl.metadata (2.2 kB)\n",
      "Collecting lxml>=3.6.0 (from newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web)\n",
      "  Using cached lxml-5.3.0-cp312-cp312-macosx_10_9_x86_64.whl.metadata (3.8 kB)\n",
      "Collecting feedparser>=5.2.1 (from newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web)\n",
      "  Using cached feedparser-6.0.11-py3-none-any.whl.metadata (2.4 kB)\n",
      "Collecting tldextract>=2.0.1 (from newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web)\n",
      "  Using cached tldextract-5.1.2-py3-none-any.whl.metadata (11 kB)\n",
      "Collecting feedfinder2>=0.0.4 (from newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web)\n",
      "  Using cached feedfinder2-0.0.4-py3-none-any.whl\n",
      "Collecting jieba3k>=0.35.1 (from newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web)\n",
      "  Using cached jieba3k-0.35.1-py3-none-any.whl\n",
      "Requirement already satisfied: python-dateutil>=2.5.3 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web) (2.9.0.post0)\n",
      "Collecting tinysegmenter==0.3 (from newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web)\n",
      "  Using cached tinysegmenter-0.3-py3-none-any.whl\n",
      "Requirement already satisfied: greenlet==3.0.3 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from playwright<2.0,>=1.30->llama-index-readers-web) (3.0.3)\n",
      "Collecting pyee==11.1.0 (from playwright<2.0,>=1.30->llama-index-readers-web)\n",
      "  Using cached pyee-11.1.0-py3-none-any.whl.metadata (2.8 kB)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from requests<3.0.0,>=2.31.0->llama-index-readers-web) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from requests<3.0.0,>=2.31.0->llama-index-readers-web) (3.6)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from requests<3.0.0,>=2.31.0->llama-index-readers-web) (2024.2.2)\n",
      "Collecting trio~=0.17 (from selenium<5.0.0,>=4.17.2->llama-index-readers-web)\n",
      "  Using cached trio-0.26.2-py3-none-any.whl.metadata (8.6 kB)\n",
      "Collecting trio-websocket~=0.9 (from selenium<5.0.0,>=4.17.2->llama-index-readers-web)\n",
      "  Using cached trio_websocket-0.11.1-py3-none-any.whl.metadata (4.7 kB)\n",
      "Collecting websocket-client~=1.8 (from selenium<5.0.0,>=4.17.2->llama-index-readers-web)\n",
      "  Using cached websocket_client-1.8.0-py3-none-any.whl.metadata (8.0 kB)\n",
      "Requirement already satisfied: six in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from feedfinder2>=0.0.4->newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web) (1.16.0)\n",
      "Collecting sgmllib3k (from feedparser>=5.2.1->newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web)\n",
      "  Using cached sgmllib3k-1.0.0-py3-none-any.whl\n",
      "Requirement already satisfied: pydantic>=1.10 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from llamaindex-py-client<0.2.0,>=0.1.16->llama-index-core<0.11.0,>=0.10.27->llama-index) (2.8.2)\n",
      "Requirement already satisfied: anyio in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from httpx->llama-index-core<0.11.0,>=0.10.27->llama-index) (4.3.0)\n",
      "Requirement already satisfied: httpcore==1.* in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from httpx->llama-index-core<0.11.0,>=0.10.27->llama-index) (1.0.5)\n",
      "Requirement already satisfied: sniffio in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from httpx->llama-index-core<0.11.0,>=0.10.27->llama-index) (1.3.1)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from httpcore==1.*->httpx->llama-index-core<0.11.0,>=0.10.27->llama-index) (0.14.0)\n",
      "Requirement already satisfied: click in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from nltk<4.0.0,>=3.8.1->llama-index-core<0.11.0,>=0.10.27->llama-index) (8.1.7)\n",
      "Requirement already satisfied: joblib in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from nltk<4.0.0,>=3.8.1->llama-index-core<0.11.0,>=0.10.27->llama-index) (1.3.2)\n",
      "Requirement already satisfied: regex>=2021.8.3 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from nltk<4.0.0,>=3.8.1->llama-index-core<0.11.0,>=0.10.27->llama-index) (2023.12.25)\n",
      "Requirement already satisfied: distro<2,>=1.7.0 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from openai>=1.14.0->llama-index-agent-openai<0.3.0,>=0.1.4->llama-index) (1.9.0)\n",
      "Requirement already satisfied: PyMuPDFb==1.24.1 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from pymupdf<2.0.0,>=1.23.21->llama-index-readers-file<0.2.0,>=0.1.4->llama-index) (1.24.1)\n",
      "Collecting requests-file>=1.4 (from tldextract>=2.0.1->newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web)\n",
      "  Using cached requests_file-2.1.0-py2.py3-none-any.whl.metadata (1.7 kB)\n",
      "Requirement already satisfied: filelock>=3.0.8 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from tldextract>=2.0.1->newspaper3k<0.3.0,>=0.2.8->llama-index-readers-web) (3.13.3)\n",
      "Collecting sortedcontainers (from trio~=0.17->selenium<5.0.0,>=4.17.2->llama-index-readers-web)\n",
      "  Using cached sortedcontainers-2.4.0-py2.py3-none-any.whl.metadata (10 kB)\n",
      "Collecting outcome (from trio~=0.17->selenium<5.0.0,>=4.17.2->llama-index-readers-web)\n",
      "  Using cached outcome-1.3.0.post0-py2.py3-none-any.whl.metadata (2.6 kB)\n",
      "Collecting wsproto>=0.14 (from trio-websocket~=0.9->selenium<5.0.0,>=4.17.2->llama-index-readers-web)\n",
      "  Using cached wsproto-1.2.0-py3-none-any.whl.metadata (5.6 kB)\n",
      "Requirement already satisfied: mypy-extensions>=0.3.0 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from typing-inspect>=0.8.0->llama-index-core<0.11.0,>=0.10.27->llama-index) (1.0.0)\n",
      "Collecting pysocks!=1.5.7,<2.0,>=1.5.6 (from urllib3[socks]<3,>=1.26->selenium<5.0.0,>=4.17.2->llama-index-readers-web)\n",
      "  Using cached PySocks-1.7.1-py3-none-any.whl.metadata (13 kB)\n",
      "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from dataclasses-json->llama-index-core<0.11.0,>=0.10.27->llama-index) (3.21.1)\n",
      "Requirement already satisfied: pytz>=2020.1 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from pandas->llama-index-core<0.11.0,>=0.10.27->llama-index) (2023.4)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from pandas->llama-index-core<0.11.0,>=0.10.27->llama-index) (2024.1)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from pydantic>=1.10->llamaindex-py-client<0.2.0,>=0.1.16->llama-index-core<0.11.0,>=0.10.27->llama-index) (0.6.0)\n",
      "Requirement already satisfied: pydantic-core==2.20.1 in /Users/sarahwooders/repos/memgpt-main/MemGPT/env/lib/python3.12/site-packages (from pydantic>=1.10->llamaindex-py-client<0.2.0,>=0.1.16->llama-index-core<0.11.0,>=0.10.27->llama-index) (2.20.1)\n",
      "Using cached llama_index_readers_web-0.1.23-py3-none-any.whl (72 kB)\n",
      "Using cached chromedriver_autoinstaller-0.6.4-py3-none-any.whl (7.6 kB)\n",
      "Using cached newspaper3k-0.2.8-py3-none-any.whl (211 kB)\n",
      "Using cached playwright-1.46.0-py3-none-macosx_11_0_universal2.whl (34.8 MB)\n",
      "Using cached pyee-11.1.0-py3-none-any.whl (15 kB)\n",
      "Downloading selenium-4.24.0-py3-none-any.whl (9.6 MB)\n",
      "   25l   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 0.0/9.6 MB ? eta -:--:--━━━╸━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 0.8/9.6 MB 23.7 MB/s eta 0:00:01━━━━━━━━━━━━━╸━━━━━━━━━━━━━━━━━━━━━━━━━━ 3.3/9.6 MB 45.3 MB/s eta 0:00:01━━━━━━━━━━━━━━━━━━━━━━━╸━━━━━━━━━━━━━━━━ 5.7/9.6 MB 52.2 MB/s eta 0:00:01━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╸━━━━━━━ 7.8/9.6 MB 54.0 MB/s eta 0:00:01━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╸ 9.6/9.6 MB 57.3 MB/s eta 0:00:01━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 9.6/9.6 MB 51.1 MB/s eta 0:00:00\n",
      "\u001b[?25hUsing cached cssselect-1.2.0-py2.py3-none-any.whl (18 kB)\n",
      "Using cached feedparser-6.0.11-py3-none-any.whl (81 kB)\n",
      "Using cached lxml-5.3.0-cp312-cp312-macosx_10_9_x86_64.whl (4.4 MB)\n",
      "Using cached tldextract-5.1.2-py3-none-any.whl (97 kB)\n",
      "Using cached trio-0.26.2-py3-none-any.whl (475 kB)\n",
      "Using cached trio_websocket-0.11.1-py3-none-any.whl (17 kB)\n",
      "Using cached websocket_client-1.8.0-py3-none-any.whl (58 kB)\n",
      "Using cached PySocks-1.7.1-py3-none-any.whl (16 kB)\n",
      "Using cached requests_file-2.1.0-py2.py3-none-any.whl (4.2 kB)\n",
      "Using cached wsproto-1.2.0-py3-none-any.whl (24 kB)\n",
      "Using cached outcome-1.3.0.post0-py2.py3-none-any.whl (10 kB)\n",
      "Using cached sortedcontainers-2.4.0-py2.py3-none-any.whl (29 kB)\n",
      "Installing collected packages: tinysegmenter, sortedcontainers, sgmllib3k, jieba3k, wsproto, websocket-client, pysocks, pyee, outcome, lxml, html2text, feedparser, cssselect, chromedriver-autoinstaller, trio, spider-client, requests-file, playwright, feedfinder2, trio-websocket, tldextract, selenium, newspaper3k, llama-index-readers-web\n",
      "  Attempting uninstall: websocket-client\n",
      "    Found existing installation: websocket-client 1.7.0\n",
      "    Uninstalling websocket-client-1.7.0:\n",
      "      Successfully uninstalled websocket-client-1.7.0\n",
      "  Attempting uninstall: html2text\n",
      "    Found existing installation: html2text 2020.1.16\n",
      "    Uninstalling html2text-2020.1.16:\n",
      "      Successfully uninstalled html2text-2020.1.16\n",
      "ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "pymemgpt 0.3.19 requires docstring-parser<0.16,>=0.15, but you have docstring-parser 0.11 which is incompatible.\n",
      "pymemgpt 0.3.19 requires html2text<2021.0.0,>=2020.1.16, but you have html2text 2024.2.26 which is incompatible.\n",
      "Successfully installed chromedriver-autoinstaller-0.6.4 cssselect-1.2.0 feedfinder2-0.0.4 feedparser-6.0.11 html2text-2024.2.26 jieba3k-0.35.1 llama-index-readers-web-0.1.23 lxml-5.3.0 newspaper3k-0.2.8 outcome-1.3.0.post0 playwright-1.46.0 pyee-11.1.0 pysocks-1.7.1 requests-file-2.1.0 selenium-4.24.0 sgmllib3k-1.0.0 sortedcontainers-2.4.0 spider-client-0.0.27 tinysegmenter-0.3 tldextract-5.1.2 trio-0.26.2 trio-websocket-0.11.1 websocket-client-1.8.0 wsproto-1.2.0\n",
      "\n",
      "[notice] A new release of pip is available: 24.0 -> 24.2\n",
      "[notice] To update, run: pip install --upgrade pip\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install llama-index llama-index-readers-web"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "79ef1d4a-0d7f-4ba5-8932-e2cdc4c65e48",
   "metadata": {},
   "outputs": [],
   "source": [
    "from memgpt.data_sources.connectors import DataConnector \n",
    "from memgpt.schemas.document import Document\n",
    "from llama_index.core import Document as LlamaIndexDocument\n",
    "from llama_index.core import SummaryIndex\n",
    "from llama_index.readers.web import SimpleWebPageReader\n",
    "\n",
    "from typing import Iterator, Tuple, Dict, List\n",
    "\n",
    "class MyCustomConnector(DataConnector): \n",
    "\n",
    "    def __init__(self, web_pages: List[str]): \n",
    "        self.web_pages = web_pages\n",
    "\n",
    "    def generate_documents(self) -> Iterator[Tuple[str, Dict]]:  \n",
    "        documents = SimpleWebPageReader(html_to_text=True).load_data(\n",
    "            self.web_pages\n",
    "        )\n",
    "        for document, web_page in zip(documents, self.web_pages): \n",
    "            metadata = {\"source_page\": web_page, \"my_metadata\": \"example\"}\n",
    "            yield document.text, metadata\n",
    "            \n",
    "    def generate_passages(\n",
    "        self, \n",
    "        documents: List[Document], \n",
    "        chunk_size: int = 1024\n",
    "    ) -> Iterator[Tuple[str, Dict]]:  # -> Iterator[Passage]:\n",
    "        from llama_index.core.node_parser import TokenTextSplitter\n",
    "\n",
    "        parser = TokenTextSplitter(chunk_size=chunk_size)\n",
    "        for document in documents:\n",
    "            llama_index_docs = [LlamaIndexDocument(text=document.text)]\n",
    "            nodes = parser.get_nodes_from_documents(llama_index_docs)\n",
    "            for node in nodes:\n",
    "                yield node.text, None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "67ab8635-d668-456f-8869-40bdaea26275",
   "metadata": {},
   "outputs": [],
   "source": [
    "web_pages = [\n",
    "    \"https://en.wikipedia.org/wiki/Memory\", \n",
    "    #\"https://en.wikipedia.org/wiki/Brain\"\n",
    "]\n",
    "\n",
    "connector = MyCustomConnector(web_pages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "fe63e6cf-e3be-437a-83cf-d58bd0e61b36",
   "metadata": {},
   "outputs": [],
   "source": [
    "source = client.create_source(\"wikipedia_brain\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b06020fd-d561-4b85-ab81-3ff7780d129d",
   "metadata": {},
   "outputs": [],
   "source": [
    "client.load_data(connector, source_name=source.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "484a2d62-8482-40e5-8bf7-91c937acd837",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MemGPT.memgpt.server.server - INFO - Created new agent from config: <memgpt.agent.Agent object at 0x14be2e960>\n"
     ]
    }
   ],
   "source": [
    "from memgpt.schemas.memory import ChatMemory\n",
    "\n",
    "wiki_persona = \"You a study assistant with a great source of knowlege \" \\\n",
    "+ \"stored in archival. You should always search your archival memory \" \\\n",
    "+ \"before responding to the human's queries. \"\n",
    "\n",
    "wiki_agent = client.create_agent(\n",
    "    name=\"wiki_agent\", \n",
    "    memory=ChatMemory(\n",
    "        human=\"Name: Sarah. Occupation: Biology PhD\", \n",
    "        persona=wiki_persona\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "5f4820c8-cac4-4906-bd2c-f63ed18620e7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MemGPT.memgpt.server.server - INFO - Grabbing agent user_id=user-552dee3c-baaf-443a-9d23-8bb54f4af964 agent_id=agent-897ef46b-2682-4d79-be8a-3ad0250ee084 from database\n",
      "MemGPT.memgpt.server.server - INFO - Creating an agent object\n",
      "MemGPT.memgpt.server.server - INFO - Adding agent to the agent cache: user_id=user-552dee3c-baaf-443a-9d23-8bb54f4af964, agent_id=agent-897ef46b-2682-4d79-be8a-3ad0250ee084\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [00:00<00:00,  4.74it/s]\n"
     ]
    }
   ],
   "source": [
    "client.attach_source_to_agent(agent_id=wiki_agent.id, source_id=source.id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "2a2730aa-d2a1-433a-bd28-fcae9a92039d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Message(id='message-f44014c3-f5aa-46b3-b64e-fbdd6e5550b6', role=<MessageRole.assistant: 'assistant'>, text=\"Let's search the archival memory to see what information I have that could help with this question.\", user_id='user-552dee3c-baaf-443a-9d23-8bb54f4af964', agent_id='agent-897ef46b-2682-4d79-be8a-3ad0250ee084', model='gpt-4', name=None, created_at=datetime.datetime(2024, 9, 3, 22, 10, 19, 834760, tzinfo=datetime.timezone.utc), tool_calls=[ToolCall(id='f9389727-5b10-4e92-bf6b-6f74c', type='function', function=ToolCallFunction(name='archival_memory_search', arguments='{\\n  \"query\": \"role of memory\",\\n  \"request_heartbeat\": true\\n}'))], tool_call_id=None),\n",
       " Message(id='message-b9c79d85-5081-4202-9d9c-5ad56c13baae', role=<MessageRole.tool: 'tool'>, text='{\\n  \"status\": \"OK\",\\n  \"message\": \"Showing 5 of 5 results (page 0/0): [\\\\n  \\\\\"timestamp: 2024-09-03 03:10:20 PM PDT-0700, memory: as a first kiss, first day of school\\\\\\\\nor first time winning a championship. These are key events in one\\'s life that\\\\\\\\ncan be remembered clearly.\\\\\\\\n\\\\\\\\nResearch suggests that declarative memory is supported by several functions of\\\\\\\\nthe medial temporal lobe system which includes the hippocampus.[49]\\\\\\\\n[Autobiographical memory](/wiki/Autobiographical_memory \\\\\\\\\\\\\"Autobiographical\\\\\\\\nmemory\\\\\\\\\\\\\") – memory for particular events within one\\'s own life – is generally\\\\\\\\nviewed as either equivalent to, or a subset of, episodic memory. [Visual\\\\\\\\nmemory](/wiki/Visual_memory \\\\\\\\\\\\\"Visual memory\\\\\\\\\\\\\") is part of memory preserving some\\\\\\\\ncharacteristics of our senses pertaining to visual experience. One is able to\\\\\\\\nplace in memory information that resembles objects, places, animals or people\\\\\\\\nin sort of a [mental image](/wiki/Mental_image \\\\\\\\\\\\\"Mental image\\\\\\\\\\\\\"). Visual memory\\\\\\\\ncan result in [priming](/wiki/Priming_\\\\\\\\\\\\\\\\(psychology\\\\\\\\\\\\\\\\) \\\\\\\\\\\\\"Priming \\\\\\\\\\\\\\\\(psychology\\\\\\\\\\\\\\\\)\\\\\\\\\\\\\")\\\\\\\\nand it is assumed some kind of perceptual representational system underlies\\\\\\\\nthis phenomenon.[49]\\\\\\\\n\\\\\\\\n#### Procedural\\\\\\\\n\\\\\\\\n[[edit](/w/index.php?title=Memory&action=edit&section=10 \\\\\\\\\\\\\"Edit section:\\\\\\\\nProcedural\\\\\\\\\\\\\")]\\\\\\\\n\\\\\\\\nIn contrast, [procedural memory](/wiki/Procedural_memory \\\\\\\\\\\\\"Procedural memory\\\\\\\\\\\\\")\\\\\\\\n(or _[implicit memory](/wiki/Implicit_memory \\\\\\\\\\\\\"Implicit memory\\\\\\\\\\\\\")_) is not\\\\\",\\\\n  \\\\\"timestamp: 2024-09-03 03:10:20 PM PDT-0700, memory: \\\\\\\\\\\\\"Attention\\\\\\\\\\\\\"). Attention plays a key role in storing information into long-term memory; without proper attention, the information might not be stored, making it impossible to be retrieved later.\\\\\\\\n\\\\\\\\n## Physiology\\\\\\\\n\\\\\\\\n[[edit](/w/index.php?title=Memory&action=edit&section=16 \\\\\\\\\\\\\"Edit section:\\\\\\\\nPhysiology\\\\\\\\\\\\\")]\\\\\\\\n\\\\\\\\nBrain areas involved in the [neuroanatomy of\\\\\\\\nmemory](/wiki/Neuroanatomy_of_memory \\\\\\\\\\\\\"Neuroanatomy of memory\\\\\\\\\\\\\") such as the\\\\\\\\n[hippocampus](/wiki/Hippocampus \\\\\\\\\\\\\"Hippocampus\\\\\\\\\\\\\"), the [amygdala](/wiki/Amygdala\\\\\\\\n\\\\\\\\\\\\\"Amygdala\\\\\\\\\\\\\"), the [striatum](/wiki/Striatum \\\\\\\\\\\\\"Striatum\\\\\\\\\\\\\"), or the [mammillary\\\\\\\\nbodies](/wiki/Mammillary_bodies \\\\\\\\\\\\\"Mammillary bodies\\\\\\\\\\\\\") are thought to be\\\\\\\\ninvolved in specific types of memory. For example, the hippocampus is believed\\\\\\\\nto be involved in spatial learning and [declarative\\\\\\\\nlearning](/wiki/Declarative_learning \\\\\\\\\\\\\"Declarative learning\\\\\\\\\\\\\"), while the\\\\\\\\namygdala is thought to be involved in [emotional\\\\\\\\nmemory](/wiki/Emotion_and_memory \\\\\\\\\\\\\"Emotion and memory\\\\\\\\\\\\\").[68]\\\\\\\\n\\\\\\\\nDamage to certain areas in patients and animal models and subsequent memory\\\\\\\\ndeficits is a primary source of information. However, rather than implicating\\\\\\\\na specific area, it could be that damage to adjacent areas, or\\\\\",\\\\n  \\\\\"timestamp: 2024-09-03 03:10:20 PM PDT-0700, memory: stored, and\\\\\\\\nretrieved when needed. It is the retention of information over time for the\\\\\\\\npurpose of influencing future action.[1] If [past\\\\\\\\nevents](/wiki/Foresight_\\\\\\\\\\\\\\\\(psychology\\\\\\\\\\\\\\\\) \\\\\\\\\\\\\"Foresight \\\\\\\\\\\\\\\\(psychology\\\\\\\\\\\\\\\\)\\\\\\\\\\\\\") could not\\\\\\\\nbe remembered, it would be impossible for language, relationships, or\\\\\\\\n[personal identity](/wiki/Personal_identity \\\\\\\\\\\\\"Personal identity\\\\\\\\\\\\\") to\\\\\\\\ndevelop.[2] Memory loss is usually described as\\\\\\\\n[forgetfulness](/wiki/Forgetting \\\\\\\\\\\\\"Forgetting\\\\\\\\\\\\\") or [amnesia](/wiki/Amnesia\\\\\\\\n\\\\\\\\\\\\\"Amnesia\\\\\\\\\\\\\").[3][4][5][6][7][8]\\\\\\\\n\\\\\\\\nMemory is often understood as an [informational\\\\\\\\nprocessing](/wiki/Information_processing_\\\\\\\\\\\\\\\\(psychology\\\\\\\\\\\\\\\\) \\\\\\\\\\\\\"Information\\\\\\\\nprocessing \\\\\\\\\\\\\\\\(psychology\\\\\\\\\\\\\\\\)\\\\\\\\\\\\\") system with explicit and implicit functioning that\\\\\\\\nis made up of a [sensory processor](/wiki/Sensory_processor \\\\\\\\\\\\\"Sensory\\\\\\\\nprocessor\\\\\\\\\\\\\"), [short-term](/wiki/Short-term_memory \\\\\\\\\\\\\"Short-term memory\\\\\\\\\\\\\") (or\\\\\\\\n[working](/wiki/Working_memory \\\\\\\\\\\\\"Working memory\\\\\\\\\\\\\")) memory, and [long-term\\\\\\\\nmemory](/wiki/Long-term_memory \\\\\\\\\\\\\"Long-term memory\\\\\\\\\\\\\").[9] This can be related to\\\\\\\\nthe [neuron](/wiki/Neuron \\\\\\\\\\\\\"Neuron\\\\\\\\\\\\\"). The sensory processor allows information\\\\\\\\nfrom the outside world to be sensed in the form of chemical and physical\\\\\\\\nstimuli and\\\\\",\\\\n  \\\\\"timestamp: 2024-09-03 03:10:20 PM PDT-0700, memory: \\\\\\\\\\\\\"False memory\\\\\\\\\\\\\") as the memories are not properly\\\\\\\\ntransferred to long-term memory. One of the primary functions of sleep is\\\\\\\\nthought to be the improvement of the consolidation of information, as several\\\\\\\\nstudies have demonstrated that memory depends on getting sufficient sleep\\\\\\\\nbetween training and test.[122] Additionally, data obtained from neuroimaging\\\\\\\\nstudies have shown activation patterns in the sleeping brain that mirror those\\\\\\\\nrecorded during the learning of tasks from the previous day,[122] suggesting\\\\\\\\nthat new memories may be solidified through such rehearsal.[123]\\\\\\\\n\\\\\\\\n## Construction for general manipulation\\\\\\\\n\\\\\\\\n[[edit](/w/index.php?title=Memory&action=edit&section=30 \\\\\\\\\\\\\"Edit section:\\\\\\\\nConstruction for general manipulation\\\\\\\\\\\\\")]\\\\\\\\n\\\\\\\\nAlthough people often think that memory operates like recording equipment,\\\\\\\\nthis is not the case. The molecular mechanisms underlying the induction and\\\\\\\\nmaintenance of memory are very dynamic and comprise distinct phases covering a\\\\\\\\ntime window from seconds to even a lifetime.[124] In fact, research has\\\\\\\\nrevealed that our memories are constructed: \\\\\\\\\\\\\"current hypotheses suggest that\\\\\\\\nconstructive processes allow individuals to simulate and imagine future\\\\\\\\nepisodes,[125] happenings, and scenarios. Since the future is not an exact\\\\\\\\nrepetition of the past, simulation of future episodes requires a complex\\\\\\\\nsystem that can draw on the past in a manner that flexibly extracts and\\\\\\\\nrecombines elements of previous experiences – a constructive rather than a\\\\\\\\nreproductive system.\\\\\\\\\\\\\"[72] People can construct their\\\\\",\\\\n  \\\\\"timestamp: 2024-09-03 03:10:20 PM PDT-0700, memory: \\\\\\\\\\\\\"Decision-making\\\\\\\\\\\\\")\\\\\\\\n  * [Problem solving](/wiki/Problem_solving \\\\\\\\\\\\\"Problem solving\\\\\\\\\\\\\")\\\\\\\\n\\\\\\\\n  \\\\\\\\n[Numerical cognition](/wiki/Numerical_cognition \\\\\\\\\\\\\"Numerical cognition\\\\\\\\\\\\\")  \\\\\\\\n  \\\\\\\\n  * [Numerosity adaptation effect](/wiki/Numerosity_adaptation_effect \\\\\\\\\\\\\"Numerosity adaptation effect\\\\\\\\\\\\\")\\\\\\\\n  * [Approximate number system](/wiki/Approximate_number_system \\\\\\\\\\\\\"Approximate number system\\\\\\\\\\\\\")\\\\\\\\n  * [Parallel individuation system](/wiki/Parallel_individuation_system \\\\\\\\\\\\\"Parallel individuation system\\\\\\\\\\\\\")\\\\\\\\n\\\\\\\\n  \\\\\\\\n  \\\\\\\\n  * [v](/wiki/Template:Cognitive \\\\\\\\\\\\\"Template:Cognitive\\\\\\\\\\\\\")\\\\\\\\n  * [t](/wiki/Template_talk:Cognitive \\\\\\\\\\\\\"Template talk:Cognitive\\\\\\\\\\\\\")\\\\\\\\n  * [e](/wiki/Special:EditPage/Template:Cognitive \\\\\\\\\\\\\"Special:EditPage/Template:Cognitive\\\\\\\\\\\\\")\\\\\\\\n\\\\\\\\n  \\\\\\\\n[![](//upload.wikimedia.org/wikipedia/commons/thumb/5/58/Memory.gif/220px-\\\\\\\\nMemory.gif)](/wiki/File:Memory.gif)Overview of the forms and functions of\\\\\\\\nmemory\\\\\\\\n\\\\\\\\n**Memory** is the faculty of the [mind](/wiki/Mind \\\\\\\\\\\\\"Mind\\\\\\\\\\\\\") by which\\\\\\\\n[data](/wiki/Data \\\\\\\\\\\\\"Data\\\\\\\\\\\\\") or [information](/wiki/Information \\\\\\\\\\\\\"Information\\\\\\\\\\\\\") is\\\\\\\\n[encoded](/wiki/Encoding_\\\\\\\\\\\\\\\\(memory\\\\\\\\\\\\\\\\) \\\\\\\\\\\\\"Encoding \\\\\\\\\\\\\\\\(memory\\\\\\\\\\\\\\\\)\\\\\\\\\\\\\"), stored, and\\\\\\\\nretrieved when needed. It is the retention of information over time for\\\\\"\\\\n]\",\\n  \"time\": \"2024-09-03 03:10:20 PM PDT-0700\"\\n}', user_id='user-552dee3c-baaf-443a-9d23-8bb54f4af964', agent_id='agent-897ef46b-2682-4d79-be8a-3ad0250ee084', model='gpt-4', name='archival_memory_search', created_at=datetime.datetime(2024, 9, 3, 22, 10, 20, 49380, tzinfo=datetime.timezone.utc), tool_calls=None, tool_call_id='f9389727-5b10-4e92-bf6b-6f74c'),\n",
       " Message(id='message-96334bf0-c2ea-49e7-a872-d6e8685e85f8', role=<MessageRole.assistant: 'assistant'>, text='Based on the details retrieved from archival memory, memory has several key roles and forms:\\n\\n1. It allows information from the past to influence future actions, making language, relationships, and personal identity develop. \\n2. There are multiple types of memory: sensory, short-term, and long-term. Each plays a key role in storing and retrieving information.\\n3. Certain areas of the brain are involved in specific types of memory. For example, the hippocampus is involved in spatial and declarative learning, while the amygdala is associated with emotional memory.\\n4. Other significant elements include the role of attention in storing information into long-term memory. Without proper attention, the information might not be stored, and it will be impossible to retrieve later. \\n5. Memory is not static, but very dynamic, covering a time window from seconds to even a lifetime. In fact, our memories are not just recordings but constructions that allow us to simulate and imagine future episodes.  \\n\\nI hope this insights provide you a good overview on the role of memory. Is there something more specific you would like to know regarding memory?', user_id='user-552dee3c-baaf-443a-9d23-8bb54f4af964', agent_id='agent-897ef46b-2682-4d79-be8a-3ad0250ee084', model='gpt-4', name=None, created_at=datetime.datetime(2024, 9, 3, 22, 10, 44, 818746, tzinfo=datetime.timezone.utc), tool_calls=[ToolCall(id='eb98aedd-bd4d-41b0-bbbe-208fd', type='function', function=ToolCallFunction(name='send_message', arguments='{\\n  \"message\": \"Based on the details retrieved from archival memory, memory has several key roles and forms:\\\\n\\\\n1. It allows information from the past to influence future actions, making language, relationships, and personal identity develop. \\\\n2. There are multiple types of memory: sensory, short-term, and long-term. Each plays a key role in storing and retrieving information.\\\\n3. Certain areas of the brain are involved in specific types of memory. For example, the hippocampus is involved in spatial and declarative learning, while the amygdala is associated with emotional memory.\\\\n4. Other significant elements include the role of attention in storing information into long-term memory. Without proper attention, the information might not be stored, and it will be impossible to retrieve later. \\\\n5. Memory is not static, but very dynamic, covering a time window from seconds to even a lifetime. In fact, our memories are not just recordings but constructions that allow us to simulate and imagine future episodes.\\\\n\\\\nI hope this insights provide you a good overview on the role of memory. Is there something more specific you would like to know regarding memory?\"\\n}'))], tool_call_id=None),\n",
       " Message(id='message-323b988f-5ba5-45d0-826d-326a64f42cf6', role=<MessageRole.tool: 'tool'>, text='{\\n  \"status\": \"OK\",\\n  \"message\": \"None\",\\n  \"time\": \"2024-09-03 03:10:44 PM PDT-0700\"\\n}', user_id='user-552dee3c-baaf-443a-9d23-8bb54f4af964', agent_id='agent-897ef46b-2682-4d79-be8a-3ad0250ee084', model='gpt-4', name='send_message', created_at=datetime.datetime(2024, 9, 3, 22, 10, 44, 819711, tzinfo=datetime.timezone.utc), tool_calls=None, tool_call_id='eb98aedd-bd4d-41b0-bbbe-208fd')]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "response = client.send_message(\n",
    "    agent_id=wiki_agent.id, \n",
    "    message=\"what is the role of memory? search archival again\", \n",
    "    role=\"user\"\n",
    ")\n",
    "response.messages"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc6352f7-1695-47e4-911b-a88c92dc4a27",
   "metadata": {},
   "source": [
    "## Connecting to external data via tools\n",
    "In the last section, we went over how to store data inside of MemGPT's archival memory. However in many cases, it can be easier to simply connect a MemGPT agent to access an external data source directly via a tool. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c34b0aaa-c4d0-41d3-b85e-aef893573576",
   "metadata": {},
   "source": [
    "Lets define a function that pretends to access an external database to lookup someone's birthday. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "76ee42bb-c111-47e0-b258-f958738b6a47",
   "metadata": {},
   "outputs": [],
   "source": [
    "def query_birthday_db(self, name: str): \n",
    "    \"\"\"\n",
    "    This tool queries an external database to \n",
    "    lookup the birthday of someone given their name.\n",
    "\n",
    "    Args: \n",
    "        name (str): The name to look up \n",
    "\n",
    "    Returns: \n",
    "        birthday (str): The birthday in mm-dd-yyyy format\n",
    "    \n",
    "    \"\"\"\n",
    "    my_fake_data = {\n",
    "        \"bob\": \"03-06-1997\", \n",
    "        \"sarah\": \"03-06-1997\"\n",
    "    } \n",
    "    name = name.lower() \n",
    "    if name not in my_fake_data: \n",
    "        return None\n",
    "    else: \n",
    "        return my_fake_data[name]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "73de8d11-6844-4dee-b2f6-1d5bc775ab19",
   "metadata": {},
   "source": [
    "### Adding a custom tool to MemGPT \n",
    "We can access this external data via an agent by adding the function as a tool to MemGPT. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "220fdbef-4c81-4964-9264-c32bca25fb9a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Tool(description=None, source_type='python', module=None, user_id='user-552dee3c-baaf-443a-9d23-8bb54f4af964', id='tool-7559f3f1-e988-4363-a1dd-2dfff8d91a64', name='query_birthday_db', tags=['extras'], source_code='def query_birthday_db(self, name: str): \\n    \"\"\"\\n    This tool queries an external database to \\n    lookup the birthday of someone given their name.\\n\\n    Args: \\n        name (str): The name to look up \\n\\n    Returns: \\n        birthday (str): The birthday in mm-dd-yyyy format\\n\\n    \"\"\"\\n    my_fake_data = {\\n        \"bob\": \"03-06-1997\", \\n        \"sarah\": \"03-06-1997\"\\n    } \\n    name = name.lower() \\n    if name not in my_fake_data: \\n        return None\\n    else: \\n        return my_fake_data[name]\\n', json_schema={'name': 'query_birthday_db', 'description': 'This tool queries an external database to ', 'parameters': {'type': 'object', 'properties': {'name': {'type': 'string', 'description': 'The name to look up '}, 'request_heartbeat': {'type': 'boolean', 'description': \"Request an immediate heartbeat after function execution. Set to 'true' if you want to send a follow-up message or run a follow-up function.\"}}, 'required': ['name', 'request_heartbeat']}})"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tool = client.create_tool(query_birthday_db, tags=[\"extras\"])\n",
    "tool"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b5db88d7-e65a-4990-8228-5f85da729392",
   "metadata": {},
   "source": [
    "We can include the tool name as an extra tool when creating an agent: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "8792ecc2-a69b-4e31-9976-4059fde9481b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MemGPT.memgpt.server.server - INFO - Created new agent from config: <memgpt.agent.Agent object at 0x14c0c79e0>\n"
     ]
    }
   ],
   "source": [
    "agent_state = client.create_agent(\n",
    "    name=\"birthday_agent\", \n",
    "    tools=[tool.name], \n",
    "    memory=ChatMemory(\n",
    "        human=\"My name is Sarah\", \n",
    "        persona=\"You are a agent with access to a birthday_db \" \\\n",
    "        + \"that you use to lookup information about users' birthdays.\"\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "fb877a44-d750-4648-ae46-58275f6305c4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MemGPT.memgpt.server.server - INFO - Grabbing agent user_id=user-552dee3c-baaf-443a-9d23-8bb54f4af964 agent_id=agent-f207e43b-2021-45be-9dde-48822c898e77 from database\n",
      "MemGPT.memgpt.server.server - INFO - Creating an agent object\n",
      "MemGPT.memgpt.server.server - INFO - Adding agent to the agent cache: user_id=user-552dee3c-baaf-443a-9d23-8bb54f4af964, agent_id=agent-f207e43b-2021-45be-9dde-48822c898e77\n",
      "[Message(id='message-d9b432de-2bb6-4c85-8bb9-a31067e271fc', role=<MessageRole.assistant: 'assistant'>, text=\"Let's access the birthday_db and find out Sarah's birthday.\", user_id='user-552dee3c-baaf-443a-9d23-8bb54f4af964', agent_id='agent-f207e43b-2021-45be-9dde-48822c898e77', model='gpt-4', name=None, created_at=datetime.datetime(2024, 9, 3, 22, 11, 24, 961893, tzinfo=datetime.timezone.utc), tool_calls=[ToolCall(id='cad6f053-27d7-4281-a04b-05a57', type='function', function=ToolCallFunction(name='query_birthday_db', arguments='{\\n  \"name\": \"Sarah\",\\n  \"request_heartbeat\": true\\n}'))], tool_call_id=None),\n",
      " Message(id='message-f27fd0a8-be72-457c-8b3c-849818aeec4d', role=<MessageRole.tool: 'tool'>, text='{\\n  \"status\": \"OK\",\\n  \"message\": \"03-06-1997\",\\n  \"time\": \"2024-09-03 03:11:24 PM PDT-0700\"\\n}', user_id='user-552dee3c-baaf-443a-9d23-8bb54f4af964', agent_id='agent-f207e43b-2021-45be-9dde-48822c898e77', model='gpt-4', name='query_birthday_db', created_at=datetime.datetime(2024, 9, 3, 22, 11, 24, 962306, tzinfo=datetime.timezone.utc), tool_calls=None, tool_call_id='cad6f053-27d7-4281-a04b-05a57'),\n",
      " Message(id='message-7423c90e-822f-40ac-aff9-8791a360dd31', role=<MessageRole.assistant: 'assistant'>, text=\"I found the information. Now, let's communicate this back to Sarah in a friendly and human-like manner.\", user_id='user-552dee3c-baaf-443a-9d23-8bb54f4af964', agent_id='agent-f207e43b-2021-45be-9dde-48822c898e77', model='gpt-4', name=None, created_at=datetime.datetime(2024, 9, 3, 22, 11, 29, 400783, tzinfo=datetime.timezone.utc), tool_calls=[ToolCall(id='1abfa1e3-a266-48a3-8773-d6087', type='function', function=ToolCallFunction(name='send_message', arguments='{\\n  \"message\": \"Hello Sarah, your birthday is on the 6th of March, 1997. Isn\\'t it wonderful to celebrate another year of life?\"\\n}'))], tool_call_id=None),\n",
      " Message(id='message-a0295ddf-29b1-44d4-bf75-7b6b249d79f7', role=<MessageRole.tool: 'tool'>, text='{\\n  \"status\": \"OK\",\\n  \"message\": \"None\",\\n  \"time\": \"2024-09-03 03:11:29 PM PDT-0700\"\\n}', user_id='user-552dee3c-baaf-443a-9d23-8bb54f4af964', agent_id='agent-f207e43b-2021-45be-9dde-48822c898e77', model='gpt-4', name='send_message', created_at=datetime.datetime(2024, 9, 3, 22, 11, 29, 401789, tzinfo=datetime.timezone.utc), tool_calls=None, tool_call_id='1abfa1e3-a266-48a3-8773-d6087')]\n"
     ]
    }
   ],
   "source": [
    "response = client.send_message(\n",
    "    agent_id=agent_state.id, \n",
    "    message = \"When is my birthday?\", \n",
    "    role = \"user\"\n",
    ") \n",
    "pprint(response.messages)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "memgpt",
   "language": "python",
   "name": "memgpt"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
